bitkeeper revision 1.634 (3fc267950JCnZdSjqz12f7QhAx9gWA)

Many files:
  Cleanups to page reference counting in Xen.

diff --git a/xen/arch/i386/mm.c b/xen/arch/i386/mm.c
index a51ac43a238c7effbe412d3fefad3dc2ee356f18..5df703de7ade7b331b11a81fbe39dde37f01ad8c 100644 (file)
--- a/xen/arch/i386/mm.c
+++ b/xen/arch/i386/mm.c
@@ -232,7 +232,7 @@ long set_gdt(struct task_struct *p,
 
         if ( (page->flags & PG_type_mask) != PGT_gdt_page )
         {
-            if ( page->type_count != 0 )
+            if ( page_type_count(page) != 0 )
                 goto out;
 
             /* Check all potential GDT entries in the page. */
@@ -253,7 +253,7 @@ long set_gdt(struct task_struct *p,
         page = frame_table + pfn;
         ASSERT((page->flags & PG_type_mask) == PGT_gdt_page);
         ASSERT((page->flags & PG_domain_mask) == p->domain);
-        ASSERT((page->type_count != 0) && (page->tot_count != 0));
+        ASSERT((page_type_count(page) != 0) && (page_tot_count(page) != 0));
         put_page_type(page);
         put_page_tot(page);
     }
@@ -340,7 +340,7 @@ long do_update_descriptor(
     case PGT_writeable_page:
         break;
     default:
-        if ( page->type_count != 0 )
+        if ( page_type_count(page) != 0 )
             goto out;
     }
 

diff --git a/xen/common/dom0_ops.c b/xen/common/dom0_ops.c
index 8db4b5fedbc720b44091c864cf58bc5e9b914b3b..344a65f83a906d6eee3ac21ccd2d17c1b3bdbcaf 100644 (file)
--- a/xen/common/dom0_ops.c
+++ b/xen/common/dom0_ops.c
@@ -344,7 +344,7 @@ long do_dom0_op(dom0_op_t *u_dom0_op)
             op.u.getpageframeinfo.domain = page->flags & PG_domain_mask;
             op.u.getpageframeinfo.type   = NONE;
 
-            if ( page->type_count != 0 )
+            if ( page_type_count(page) != 0 )
             {
                 switch ( page->flags & PG_type_mask )
                 {

diff --git a/xen/common/dom_mem_ops.c b/xen/common/dom_mem_ops.c
index 2e8696c53309ca765c884ab426e365a978626e74..c8869882aef86ba8f8e71151b10c4d4e5ce5d5e7 100644 (file)
--- a/xen/common/dom_mem_ops.c
+++ b/xen/common/dom_mem_ops.c
@@ -57,7 +57,8 @@ static long alloc_dom_mem(struct task_struct *p, reservation_increase_t op)
         /* Get a free page and add it to the domain's page list. */
         pf = list_entry(temp, struct pfn_info, list);
         pf->flags |= p->domain;
-        pf->type_count = pf->tot_count = 0;
+        set_page_type_count(pf, 0);
+        set_page_tot_count(pf, 0);
         temp = temp->next;
         list_del(&pf->list);
         list_add_tail(&pf->list, &p->pg_head);
@@ -109,12 +110,13 @@ static long free_dom_mem(struct task_struct *p, reservation_decrease_t op)
         }
 
         pf = &frame_table[mpfn];
-        if ( (pf->type_count != 0) || 
-             (pf->tot_count != 0) ||
+        if ( (page_type_count(pf) != 0) || 
+             (page_tot_count(pf) != 0) ||
              ((pf->flags & PG_domain_mask) != p->domain) )
         {
             DPRINTK("Bad page free for domain %d (%ld, %ld, %08lx)\n",
-                    p->domain, pf->type_count, pf->tot_count, pf->flags);
+                    p->domain, page_type_count(pf), 
+                    page_tot_count(pf), pf->flags);
             rc = -EINVAL;
             goto out;
         }

diff --git a/xen/common/domain.c b/xen/common/domain.c
index 1c1a25819b029fb4d7f55eb9054ee986f8f3d76e..eae232206bac669cc7c2356e2ee65aff2a3d6f7c 100644 (file)
--- a/xen/common/domain.c
+++ b/xen/common/domain.c
@@ -245,7 +245,8 @@ unsigned int alloc_new_dom_mem(struct task_struct *p, unsigned int kbytes)
     {
         pf = list_entry(temp, struct pfn_info, list);
         pf->flags = p->domain;
-        pf->type_count = pf->tot_count = 0;
+        set_page_type_count(pf, 0);
+        set_page_tot_count(pf, 0);
         temp = temp->next;
         list_del(&pf->list);
         list_add_tail(&pf->list, &p->pg_head);
@@ -273,7 +274,9 @@ void free_all_dom_mem(struct task_struct *p)
     while ( (ent = p->pg_head.next) != &p->pg_head )
     {
         struct pfn_info *pf = list_entry(ent, struct pfn_info, list);
-        pf->type_count = pf->tot_count = pf->flags = 0;
+        set_page_type_count(pf, 0);
+        set_page_tot_count(pf, 0);
+        pf->flags = 0;
         ASSERT(ent->next->prev == ent);
         ASSERT(ent->prev->next == ent);
         list_del(ent);
@@ -513,7 +516,8 @@ int setup_guestos(struct task_struct *p, dom0_createdomain_t *params,
         
         page = frame_table + (cur_address >> PAGE_SHIFT);
         page->flags = dom | PGT_writeable_page | PG_need_flush;
-        page->type_count = page->tot_count = 1;
+        set_page_type_count(page, 1);
+        set_page_tot_count(page, 1);
         /* Set up the MPT entry. */
         machine_to_phys_mapping[cur_address >> PAGE_SHIFT] = count;
 
@@ -535,7 +539,7 @@ int setup_guestos(struct task_struct *p, dom0_createdomain_t *params,
         *l1tab = mk_l1_pgentry(l1_pgentry_val(*l1tab) & ~_PAGE_RW);
         page = frame_table + l1_pgentry_to_pagenr(*l1tab);
         page->flags = dom | PGT_l1_page_table;
-        page->tot_count++;
+        get_page_tot(page);
         l1tab++;
         if( !((unsigned long)l1tab & (PAGE_SIZE - 1)) )
         {
@@ -544,9 +548,9 @@ int setup_guestos(struct task_struct *p, dom0_createdomain_t *params,
             l2tab++;
         }
     }
-    page->type_count |= REFCNT_PIN_BIT;
-    page->tot_count  |= REFCNT_PIN_BIT;
-    page->flags = dom | PGT_l2_page_table;
+    get_page_type(page); /* guest_pinned */
+    get_page_tot(page);  /* guest_pinned */
+    page->flags = dom | PG_guest_pinned | PGT_l2_page_table;
     unmap_domain_mem(l1start);
 
     /* Set up shared info area. */

diff --git a/xen/common/memory.c b/xen/common/memory.c
index 01c846542ac30f92092efee59c31c09caaf03945..77fe5822a25080427bb8ca43c815dc77f7d7bc2e 100644 (file)
--- a/xen/common/memory.c
+++ b/xen/common/memory.c
@@ -33,7 +33,7 @@
  * physical page frame by a domain, including uses as a page directory,
  * a page table, or simple mappings via a PTE. This count prevents a
  * domain from releasing a frame back to the hypervisor's free pool when
- * it is still referencing it!
+ * it still holds a reference to it.
  * 
  * TYPE_COUNT is more subtle. A frame can be put to one of three
  * mutually-exclusive uses: it might be used as a page directory, or a
@@ -140,7 +140,9 @@
 #include <asm/domain_page.h>
 
 #if 0
-#define MEM_LOG(_f, _a...) printk("DOM%d: (file=memory.c, line=%d) " _f "\n", current->domain, __LINE__, ## _a )
+#define MEM_LOG(_f, _a...) 
+  printk("DOM%d: (file=memory.c, line=%d) " _f "\n", \
+         current->domain, __LINE__, ## _a )
 #else
 #define MEM_LOG(_f, _a...) ((void)0)
 #endif
@@ -230,7 +232,7 @@ static void __invalidate_shadow_ldt(struct task_struct *p)
         page = frame_table + pfn;
         ASSERT((page->flags & PG_type_mask) == PGT_ldt_page);
         ASSERT((page->flags & PG_domain_mask) == p->domain);
-        ASSERT((page->type_count != 0) && (page->tot_count != 0));
+        ASSERT((page_type_count(page) != 0) && (page_tot_count(page) != 0));
         put_page_type(page);
         put_page_tot(page);                
     }
@@ -271,7 +273,7 @@ int map_ldt_shadow_page(unsigned int off)
     page = frame_table + (l1e >> PAGE_SHIFT);
     if ( unlikely((page->flags & PG_type_mask) != PGT_ldt_page) )
     {
-        if ( unlikely(page->type_count != 0) )
+        if ( unlikely(page_type_count(page) != 0) )
             goto out;
 
         /* Check all potential LDT entries in the page. */
@@ -367,7 +369,7 @@ static int dec_page_refcnt(unsigned long page_nr, unsigned int type)
                 type);
         return -1;
     }
-    ASSERT((page_type_count(page) & ~REFCNT_PIN_BIT) != 0);
+    ASSERT(page_type_count(page) != 0);
     put_page_tot(page);
     return put_page_type(page);
 }
@@ -607,7 +609,7 @@ static void put_page(unsigned long page_nr, int writeable)
     page = frame_table + page_nr;
     ASSERT(DOMAIN_OKAY(page->flags));
     ASSERT((!writeable) || 
-           (((page_type_count(page) & ~REFCNT_PIN_BIT) != 0) && 
+           ((page_type_count(page) != 0) && 
             ((page->flags & PG_type_mask) == PGT_writeable_page) &&
             ((page->flags & PG_need_flush) == PG_need_flush)));
     if ( writeable )
@@ -735,7 +737,7 @@ static int do_extended_command(unsigned long ptr, unsigned long val)
     switch ( cmd )
     {
     case MMUEXT_PIN_L1_TABLE:
-        if ( unlikely(page->type_count & REFCNT_PIN_BIT) )
+        if ( unlikely(page->flags & PG_guest_pinned) )
         {
             MEM_LOG("Pfn %08lx already pinned", pfn);
             err = 1;
@@ -745,7 +747,7 @@ static int do_extended_command(unsigned long ptr, unsigned long val)
         goto mark_as_pinned;
 
     case MMUEXT_PIN_L2_TABLE:
-        if ( unlikely(page->type_count & REFCNT_PIN_BIT) )
+        if ( unlikely(page->flags & PG_guest_pinned) )
         {
             MEM_LOG("Pfn %08lx already pinned", pfn);
             err = 1;
@@ -759,25 +761,19 @@ static int do_extended_command(unsigned long ptr, unsigned long val)
             MEM_LOG("Error while pinning pfn %08lx", pfn);
             break;
         }
-        put_page_type(page);
-        put_page_tot(page);
-        page->type_count |= REFCNT_PIN_BIT;
-        page->tot_count  |= REFCNT_PIN_BIT;
+        page->flags |= PG_guest_pinned;
         break;
 
     case MMUEXT_UNPIN_TABLE:
-        if ( !DOMAIN_OKAY(page->flags) )
+        if ( unlikely(!DOMAIN_OKAY(page->flags)) )
         {
             err = 1;
             MEM_LOG("Page %08lx bad domain (dom=%ld)",
                     ptr, page->flags & PG_domain_mask);
         }
-        else if ( (page->type_count & REFCNT_PIN_BIT) )
+        else if ( likely(page->flags & PG_guest_pinned) )
         {
-            page->type_count &= ~REFCNT_PIN_BIT;
-            page->tot_count  &= ~REFCNT_PIN_BIT;
-            get_page_type(page);
-            get_page_tot(page);
+            page->flags &= ~PG_guest_pinned;
             ((page->flags & PG_type_mask) == PGT_l1_page_table) ?
                 put_l1_table(pfn) : put_l2_table(pfn);
         }
@@ -916,7 +912,7 @@ int do_mmu_update(mmu_update_t *ureqs, int count)
                                        mk_l2_pgentry(req.val)); 
                     break;                    
                 default:
-                    if ( page->type_count == 0 )
+                    if ( page_type_count(page) == 0 )
                     {
                         *(unsigned long *)req.ptr = req.val;
                         err = 0;

diff --git a/xen/drivers/block/xen_block.c b/xen/drivers/block/xen_block.c
index dc7a790408ef08f98d120ca34ff23b3832f00551..78ba3c86b9ff114390a9d6f6a4999dc099107e17 100644 (file)
--- a/xen/drivers/block/xen_block.c
+++ b/xen/drivers/block/xen_block.c
@@ -350,7 +350,7 @@ static int __buffer_is_valid(struct task_struct *p,
         /* If reading into the frame, the frame must be writeable. */
         if ( writeable_buffer &&
              ((page->flags & PG_type_mask) != PGT_writeable_page) &&
-             (page->type_count != 0) )
+             (page_type_count(page) != 0) )
         {
             DPRINTK("non-writeable page passed for block read\n");
             goto out;
@@ -376,7 +376,7 @@ static void __lock_buffer(unsigned long buffer,
         page = frame_table + pfn;
         if ( writeable_buffer )
         {
-            if ( page->type_count == 0 )
+            if ( page_type_count(page) == 0 )
             {
                 page->flags &= ~PG_type_mask;
                 /* No need for PG_need_flush here. */

diff --git a/xen/include/xeno/mm.h b/xen/include/xeno/mm.h
index d565583d6ad8e96674e85961db6845b6fcba4eb4..8f0c0323670ac8dcc1f5c3ba72bc16a51cbad880 100644 (file)
--- a/xen/include/xeno/mm.h
+++ b/xen/include/xeno/mm.h
@@ -59,13 +59,6 @@ typedef struct pfn_info {
     unsigned long type_count;   /* pagetable/dir, or domain-writeable refs. */
 } frame_table_t;
 
-/*
- * We use a high bit to indicate that a page is pinned.
- * We do not use the top bit as that would mean that we'd get confused with
- * -ve error numbers in some places in common/memory.c.
- */
-#define REFCNT_PIN_BIT 0x40000000UL
-
 #define get_page_tot(p)                 ((p)->tot_count++)
 #define put_page_tot(p)                 \
     ({ ASSERT((p)->tot_count != 0); --(p)->tot_count; })
@@ -83,9 +76,9 @@ typedef struct pfn_info {
 #define PG_slab               24
 /* domain flags (domain != 0) */
 /*
- * NB. The following three flags are MUTUALLY EXCLUSIVE!
+ * NB. The following page types are MUTUALLY EXCLUSIVE.
  * At most one can be true at any point, and 'type_count' counts how many
- * references exist of teh current type. A change in type can only occur
+ * references exist of the current type. A change in type can only occur
  * when type_count == 0.
  */
 #define PG_type_mask        (15<<24) /* bits 24-27 */
@@ -111,6 +104,13 @@ typedef struct pfn_info {
  */
 #define PG_need_flush       (1<<28)
 
+/*
+ * This bit indicates that the guest OS has pinned the page to its current
+ * type. For page tables this can avoid the frame scanning and reference-count
+ * updates that occur when the type count falls to zero.
+ */
+#define PG_guest_pinned     (1<<29)
+
 #define PageSlab(page)         test_bit(PG_slab, &(page)->flags)
 #define PageSetSlab(page)      set_bit(PG_slab, &(page)->flags)
 #define PageClearSlab(page)    clear_bit(PG_slab, &(page)->flags)
@@ -118,11 +118,16 @@ typedef struct pfn_info {
 #define SHARE_PFN_WITH_DOMAIN(_pfn, _dom)                            \
     do {                                                             \
         (_pfn)->flags = (_dom) | PGT_writeable_page | PG_need_flush; \
-        (_pfn)->tot_count = (_pfn)->type_count = 2;                  \
+        set_page_tot_count((_pfn), 2);                               \
+        set_page_type_count((_pfn), 2);                              \
     } while ( 0 )
 
-#define UNSHARE_PFN(_pfn) \
-    (_pfn)->flags = (_pfn)->type_count = (_pfn)->tot_count = 0
+#define UNSHARE_PFN(_pfn)                                            \
+    do {                                                             \
+        (_pfn)->flags = 0;                                           \
+        set_page_tot_count((_pfn), 0);                               \
+        set_page_type_count((_pfn), 0);                              \
+    } while ( 0 )
 
 /* The array of struct pfn_info,  
  * free pfn list and number of free pfns in the free list

diff --git a/xen/net/dev.c b/xen/net/dev.c
index e7b9f2d01ccea477944be068136896b3e5641c64..280db4def1e91f97e3074182322ae6ad7418d247 100644 (file)
--- a/xen/net/dev.c
+++ b/xen/net/dev.c
@@ -550,7 +550,8 @@ void deliver_packet(struct sk_buff *skb, net_vif_t *vif)
     }
 
     /* Give the new page to the domain, marking it writeable. */
-    new_page->tot_count = new_page->type_count = 1;
+    set_page_type_count(new_page, 1);
+    set_page_tot_count(new_page, 1);
     new_page->flags = vif->domain->domain | PGT_writeable_page | PG_need_flush;
     list_add(&new_page->list, &vif->domain->pg_head);
     
@@ -2113,10 +2114,10 @@ static long get_bufs_from_vif(net_vif_t *vif)
 
         if ( ((buf_page->flags & (PG_type_mask | PG_domain_mask)) !=
               (PGT_writeable_page | p->domain)) || 
-             (buf_page->tot_count != 1) )
+             (page_tot_count(buf_page) != 1) )
         {
             DPRINTK("Need a mapped-once writeable page (%ld/%ld/%08lx)\n",
-                    buf_page->type_count, buf_page->tot_count, 
+                    page_type_count(buf_page), page_tot_count(buf_page), 
                     buf_page->flags);
             make_rx_response(vif, rx.id, 0, RING_STATUS_BAD_PAGE, 0);
             goto rx_unmap_and_continue;
@@ -2129,7 +2130,9 @@ static long get_bufs_from_vif(net_vif_t *vif)
         get_page_type(pte_page);
         get_page_tot(pte_page);
         *ptep &= ~_PAGE_PRESENT;
-        buf_page->flags = buf_page->type_count = buf_page->tot_count = 0;
+        buf_page->flags = 0;
+        set_page_type_count(buf_page, 0);
+        set_page_tot_count(buf_page, 0);
         list_del(&buf_page->list);
 
         vif->rx_shadow_ring[j].id          = rx.id;
@@ -2198,7 +2201,8 @@ long flush_bufs_for_vif(net_vif_t *vif)
             *pte = (rx->buf_pfn<<PAGE_SHIFT) | (*pte & ~PAGE_MASK) | 
                 _PAGE_RW | _PAGE_PRESENT;
             page->flags |= PGT_writeable_page | PG_need_flush;
-            page->type_count = page->tot_count = 1;
+            set_page_type_count(page, 1);
+            set_page_tot_count(page, 1);
         }
         unmap_domain_mem(pte);
 

diff --git a/xen/net/skbuff.c b/xen/net/skbuff.c
index 4e07cb93821675d758dd1ee53de8b9e55771ea7b..d8950633b952153b5c4c3d867743e1f62fcf1482 100644 (file)
--- a/xen/net/skbuff.c
+++ b/xen/net/skbuff.c
@@ -162,7 +162,9 @@ static inline void dealloc_skb_data_page(struct sk_buff *skb)
 
     spin_lock_irqsave(&free_list_lock, flags);
         
-    pf->flags = pf->type_count = pf->tot_count = 0;
+    pf->flags = 0;
+    set_page_type_count(pf, 0);
+    set_page_tot_count(pf, 0);
     list_add(&pf->list, &free_list);
     free_pfns++;